#!/usr/bin/env python3
# Author: Joshua Chen
# Date: 2016-01-03
# Location: Shenzhen
# Description: a log tool with encrypted remote backup.

import sys, os
import re
import time
import sqlite3
import tempfile

prog_path = os.path.realpath(__file__)
prog_dir  = os.path.dirname(prog_path)
lib_dir   = os.path.join(prog_dir, 'lib')
sys.path.insert(0, lib_dir)

from config import Config
from log import Log
from record import Record
from timeutils import isodatetime, isodate
from sqlitestorage import SqliteStorage
from xmlstorage import XmlStorage
import interact
import applib


def failure_handler(data):
    """For called by the log adding/editing method,
    in case of failure, to prevent data loss.
    """
    tmpfile = tempfile.NamedTemporaryFile(delete=False)
    tmpfile.write(data.encode())
    tmpfile.close()
    msg = 'operation failed, data stored here: %s' % tmpfile.name
    print(msg, file=sys.stderr)


class App:
    def __init__(self, configs):
        self.configs = configs

    def extractLimitArgs(self, args):
        """ Get all limit arguments out of the args, return the last one.
        """
        argsCopy = args[:]
        limit    = None
        for arg in argsCopy:
            if re.search('^-[1-9][0-9]*$', arg):
                limit = arg
                args.remove(arg)
        if limit:
            limit = int(limit[1:])
        return limit


    def procArgs(self, args):
        """ Process the arguments, return a filter function.
        -t is for the 'time' field of the record,
        -mt is for the 'mtime' field, it ok to specify
        multiple time options in the command line, but
        if -t and -mt are used together, the last one
        of them determines the time type, that is:
            $ cmd list -t -1 -mt -2
        is equivalent of
            $ cmd list -mt -1 -mt -2
        """
        allMatch   = False
        ascending  = True
        patterns   = []
        times      = []
        timeField  = 'time'
        remainOpts = []
        fmt        = None
        orderBy    = None
        while args:
            arg = args.pop(0)
            if arg[:2] == '-S':     # regular expression
                if len(arg) > 2:
                    regexp = arg[2:]
                else:
                    assert len(args) > 0, "need argument for -S option"
                    regexp = args.pop(0)
                pattern = applib.parsePattern(regexp)
                patterns.append(pattern)
            elif arg == '--all-match':  # affects the RE evaluation
                allMatch = True
            elif arg[:6] == '--sort':
                if arg.startswith('--sort='):
                    orderBy = arg[7:]
                else:
                    assert len(args) > 0, "need argument for --sort option"
                    orderBy = args.pop(0)
                orderBy = orderBy.lower()
                applib.checkFieldName(orderBy)
            elif arg == '-r':
                ascending = False
            elif arg[:2] == '-t':   # time
                if len(arg) > 2:
                    timeArg = arg[2:]
                else:
                    assert len(args) > 0, "need argument for -t option"
                    timeArg = args.pop(0)
                assert applib.validateTime(timeArg), ("invalid time: %s" % timeArg)
                times.extend(applib.parseTime(timeArg))
                timeField = 'time'
            elif arg[:3] == '-mt':  # mtime
                if len(arg) > 3:
                    timeArg = arg[3:]
                else:
                    assert len(args) > 0, "need argument for -mt option"
                    timeArg = args.pop(0)
                assert applib.validateTime(timeArg), ("invalid time: %s" % timeArg)
                times.extend(applib.parseTime(timeArg))
                timeField = 'mtime'
            elif arg[:2] == '-f':   # format
                if len(arg) > 2:
                    fmt = arg[2:]
                else:
                    assert len(args) > 0, "need argument for -f option"
                    fmt = args.pop(0)
            elif re.search('^(-[1-9][0-9]*)|([a-zA-Z0-9]{1,40})$', arg):
                remainOpts.append(arg)
            else:
                assert False, "unrecognized option: %s" % arg

        limit = self.extractLimitArgs(remainOpts)
        ids   = remainOpts
        regxs = {'allMatch': allMatch, 'patterns': patterns} if patterns else None
        times = {'field': timeField, 'points': times} if times else None
        order = {'by': orderBy, 'ascending': ascending} if orderBy else None
        res   = dict(regxs=regxs, times=times, limit=limit,
                     ids=ids, fmt=fmt, order=order)
        return res


    def simpleSearch(self, logger, args, fmt='%I:%t:%mt:%c:%p:%g:%S', engine=None):
        """ Search and return the result as a list
        Originally for edit and delete actions.
        """
        if fmt:
            args.extend(['-f', fmt])
        criteria, order, fmt = self.procSearchArgs(args)
        fields, formater = self.parseDisplayFormat(fmt)
        result  = list(logger._list(fields, criteria, order, engine=engine))
        return result


    def delete(self, args):
        """ Delete logs

        Identify logs by the ID, multiple IDs are accepted,
        either from the command line matching arguments which
        is used by the listing function, or from a file; the
        file shall have one line for one ID.
        """
        if '--help' in args:
            help('del')
            exit(0)

        assert len(args) > 0, "wrong arguments"
        sOpts = []
        ifile = None
        force = False
        while args:
            arg = args.pop(0)
            if arg[:2] == '-i':     # input file
                if len(arg) > 2:
                    ifile = arg[2:]
                else:
                    assert len(args) > 0, "need argument for -i option"
                    ifile = args.pop(0)
            elif arg == '-f':
                force = True
            else:   # log ID
                sOpts.append(arg)

        logger = Log(self.configs)
        result = self.simpleSearch(logger, sOpts)
        ids    = [x['id'] for x in result]
        if ifile:
            lines = open(ifile).readlines()
            lines = [x.rstrip() for x in lines if len(x) > 1]
            ids.extend(lines)
        assert ids, "no IDs provided/matched"
        ids    = set(ids)
        logger.delete(ids, force=force)


    def edit(self, args):
        """ Edit a log
        IDs are acquired by the command line matching
        arguments, which is used for the listing function.
        """
        if '--help' in args:
            help('edit')
            exit(0)

        assert len(args) > 0, "wrong arguments"
        logger  = Log(self.configs)
        result  = self.simpleSearch(logger, args)

        ids = []
        if len(result) == 1:
            ids.append(result[0]['id'])
        else:
            class E:
                def __init__(self, data):
                    self.data = data
                def __repr__(self):
                    D     = self.data
                    keys  = ['Time', 'MTime', 'Scene', 'People', 'Tag']
                    conv  = lambda x: x
                    funcs = [isodatetime, isodatetime] + [conv] * 3
                    text  = Record.formatFields(keys, funcs, D)
                    text  = Record.formatPrependId(text, D)
                    text  += '\n%s' % D['subject'][:40]
                    text  = text.replace('\n', '\n  ')
                    return text
            x = [E(r) for r in result]
            idx, picked = interact.printAndPick(x, lineMode=True, default=-1)
            if picked:  # user picked one
                ids.append(picked.data['id'])
            else:       # user just press Enter, means pick ALL.
                ids = [x['id'] for x in result]

        for id in ids:
            logger.edit(id, fail_callback=failure_handler)

    def push(self, args):
        """ Sync with the git server
        """
        if '--help' in args:
            help('push')
            exit(0)

        logger     = Log(self.configs)
        remotes    = []
        allRemotes = False
        while args:
            arg = args.pop(0)
            if arg == '-a':
                remotes    = logger.git.allRemotes()
                allRemotes = True
                break
            else:
                remotes.append(arg)
        if not remotes and not allRemotes:
            remotes = ['origin']
        for remote in remotes:
            print('pushing to "%s"' % remote)
            stat = logger.push(remote)
            if not stat:
                break
        exit(0 if stat else 1)


    def fetch(self, args):
        """ Download changes from the git server
        """
        if '--help' in args:
            help('fetch')
            exit(0)

        logger     = Log(self.configs)
        remotes    = []
        allRemotes = False
        while args:
            arg = args.pop(0)
            if arg == '-a':
                remotes    = logger.git.allRemotes()
                allRemotes = True
                break
            else:
                remotes.append(arg)
        if not remotes and not allRemotes:
            remotes = ['origin']
        for remote in remotes:
            print('fetching from "%s"' % remote)
            stat = logger.fetch(remote)
            if not stat:
                break
        exit(0 if stat else 1)


    def clone(self, args):
        """ Clone the repository from the remote
        """
        if '--help' in args:
            help('clone')
            exit(0)

        assert len(args) == 1, "wrong arguments"
        remote  = args[0]
        dataDir = self.configs['dataDir']
        stat    = os.system('git shadow-clone %s %s' % (remote, dataDir))
        exit(0 if stat else 1)


    """ Methods defined below are Record definition specific,
    subclasses shall redefine/extend these methods according
    to the Record fields definition, or add more others.
    """


    def add(self, args):
        """ Add a log
        If message provided, all lines before an empty line
        will serve as the subject, and the rest as the data.
        If the data is not available from the argument, read
        it from the stdin if the stdin is not a tty. If the
        -m option is not provided, or its argument is empty,
        enter interactive mode and open up an editor to
        collect the subject and optionally the data.

        Priority of the source of subject and data, descending:

            cmdline arguments --> stdin --> editor

        """
        if '--help' in args:
            help('add')
            exit(0)

        tag = _time = scene = people = message = subject = ''
        data = b''
        binary = False
        while len(args) >= 2:
            key = args.pop(0)
            if key == '-g':
                tag = args.pop(0)
            elif key == '-t':
                _time = args.pop(0)
            elif key == '-c':
                scene = args.pop(0)
            elif key == '-p':
                people = args.pop(0)
            elif key == '-m':
                message  = args.pop(0)
        # subject and data from command line
        if message:
            msgLines = message.rstrip('\n').split('\n\n')
            subject  = msgLines.pop(0)
            data     = '\n\n'.join(msgLines).encode()

        interactive = not subject
        if interactive and not os.isatty(sys.stdin.fileno()):
            msg = "no -m option, assume interactive mode, but stdin is not a terminal"
            raise applib.NotTerminalException(msg)

        # data from stdin
        if not data and not os.isatty(sys.stdin.fileno()):
            stdin = os.fdopen(sys.stdin.fileno(), 'rb')
            iData = stdin.read()
            if applib.isBinary(iData):
                binary = True
                data   = applib.binToAsc(iData)
            else:
                binary = False
                data   = iData
        if not binary:
            data = data.decode()
        if not _time:
            _time = isodatetime()
        logger = Log(self.configs)
        logger.add(subject=subject, time=_time, scene=scene,
                   people=people, tag=tag, data=data,
                   binary=binary, interactive=interactive,
                   fail_callback=failure_handler)

    def _list(self, args):
        """ List log summary

        Match log entries by time, and/or regular expression.

        Time format (implemented in applib.compreDay):
            today: the day when the command runs
            negative number: N-th day ago (-1, -70)
            one or two digit number: a day of a month (1, 12)
            four digit number: a day of the specific month  (0413, 0314)
            four digit number: a year  (2015, 2016)
            six digit number: a month of a year  (201512, 201601)
            eight digit number: a day of a month of a year  (20160101)

        Time contextual meaning (implemented in parseTime):
            a. pure number/word: from the first second of it,
                                 to the last second of it
            b. colon separated : from the first second of the first,
                                 to the last second of the second,
                                 'today' next to the colon can be omitted.
            c. comma separated: combination of 'a' or 'b' types
        """
        if '--help' in args:
            help('list')
            exit(0)

        criteria, order, fmt = self.procSearchArgs(args)
        logger  = Log(self.configs)
        fields, formater = self.parseDisplayFormat(fmt)
        result  = logger._list(fields, criteria, order)
        color   = False if fmt else True    # no color if display format specified
        applib.pageOut(result, formater, color)


    def procSearchArgs(self, args):
        """ Process the arguments, return the
        criteria, order, and format information.
        """
        criteria = {}   # search criteria
        order    = None # order criteria
        fmt      = None # what and how to display
        if len(args) > 0:
            x = self.procArgs(args)
            criteria['times'] = x['times']
            criteria['regxs'] = x['regxs']
            criteria['limit'] = x['limit']
            criteria['ids']   = x['ids']
            fmt   = x['fmt']
            order = x['order']
        return criteria, order, fmt


    def parseDisplayFormat(self, fmt):
        """ Parse the string 'fmt', return 'fields' which
        is the fields to collect, and fmtStr which is the
        format for displaying the result. If the fmt is
        None, collect all fields of Record, and the default
        display format applies.

        The order of 'fieldMaps' is significant when do
        the replacing, if two flags have identical beginning,
        like %t and %td, the longer shall be processed first,
        So we sort it by to the length of the key.
        """
        if not fmt:
            fields   = list(Record.fields.keys())
            formater = Record.defaultFormater
        else:
            shortId   = lambda s: s[:7]
            shortSbj  = lambda s: s[:10]
            # flag: [fieldName, flagReplacement, converter]
            fieldMaps = {
                '%i'  : ['id', 'id', shortId],
                '%I'  : ['id', 'fullId', None],
                '%s'  : ['subject', 'subject', None],
                '%S'  : ['subject', 'shortSubject', shortSbj],
                '%a'  : ['author', 'author', None],
                '%t'  : ['time', 'time', isodatetime],
                '%td' : ['time', 'dateOfTime', isodate],
                '%mt' : ['mtime', 'mtime', isodatetime],
                '%mtd': ['mtime', 'dateOfMTime', isodate],
                '%c'  : ['scene', 'scene', None],
                '%p'  : ['people', 'people', None],
                '%g'  : ['tag', 'tag', None],
                '%d'  : ['data', 'data', None],
            }
            fields   = set()
            fmtStr   = fmt
            convList = []
            key      = lambda x: len(x[0])
            fieldMaps = sorted(fieldMaps.items(), key=key, reverse=True)
            for k, desc in fieldMaps:
                if k in fmtStr:
                    field, replacement, conv = desc
                    convList.append(desc)
                    fields.add(field)
                    fmtStr = fmtStr.replace(k, '{%s}' % replacement)
            def formater(data, colorFunc=None, n=None):
                """ parameter colorFunc and 'n' are place holders
                """
                if convList:
                    for field, replacement, conv in convList:
                        if conv:
                            data[replacement] = conv(data[field])
                        else:
                            data[replacement] = data[field]
                return fmtStr.format(**data) + '\n'
        return (fields, formater)

    def man(self, args):
        """ Management function, gateway for various
        management functions, like 'unity', 'export'.
        """
        if '--help' in args:
            help('man')
            exit(0)

        assert len(args) > 0, "wrong arguments"
        func = args.pop(0)
        if func == 'unity':
            self.flushSqliteWithXml(args)
        elif func == 'export':
            self.export(args)

    def export(self, args):
        """ Export data from the sqlite engine,
        write it out as text/xml/sqlite.
        """
        listArgs = []
        format = output = None
        while args:
            arg = args.pop(0)
            if arg[:2] == '-f':
                if len(arg) > 2:
                    format = arg[2:]
                else:
                    assert len(args) > 0, "need argument for -f option"
                    format = args.pop(0)
            elif arg[:2] == '-o':
                if len(arg) > 2:
                    output = arg[2:]
                else:
                    assert len(args) > 0, "need argument for -o option"
                    output = args.pop(0)
            else:
                listArgs.append(arg)
        assert format and output, "both -f and -o are required"
        assert not os.path.exists(output), "%s already exists" % output
        msg = 'export as %s, to %s, confirm? [y/N] ' % (format, output)
        ans = interact.readstr(msg, default='n')
        if ans == 'n':
            return
        logger = Log(self.configs)
        result = self.simpleSearch(logger, listArgs, fmt=None, engine=SqliteStorage)
        if format == 'sqlite':
            conn   = sqlite3.connect(output)
            self.recordsToSqlite(conn, result)
        elif format == 'xml':
            self.recordsToXml(output, result)
        elif format == 'text':
            self.recordsToText(output, result)

    def recordsToXml(self, dir, records):
        """ Writes the record data as xml files
        to the directory 'dir', records is a list
        of dict objects.
        """
        for record in records:
            XmlStorage.saveRecord(record, dir=dir)

    def recordsToText(self, dir, records):
        """ Writes the record data as xml files
        to the directory 'dir', records is a list
        of dict objects.
        """
        for record in records:
            dateEle    = isodate(record['time']).split('-')
            absDirPath = os.path.join(dir, *dateEle)
            os.makedirs(absDirPath, exist_ok=True)
            path = os.path.join(absDirPath, record['id'])
            code = Record.defaultFormater(record, (lambda x: x), n=0)
            open(path, 'w').write(code)

    def flushSqliteWithXml(self, args):
        """ Remove all data in the sqlite storage, and export all
        records from the xml storage into the sqlite storage, thus,
        the sqlite storage data can be synchronized after a manual
        Git operation. Defaults to export all records, and filter-
        ing options like the listing options are accepted.
        """
        logger = Log(self.configs)
        # all fields shall be fetched, so we ignore user's -f options
        assert '-f' not in args, '-f option is forbidden'
        result = self.simpleSearch(logger, args, fmt=None, engine=XmlStorage)
        self.recordsToSqlite(SqliteStorage.conn, result)

    def recordsToSqlite(self, conn, records):
        """ Drop the sqlite database table, insert
        all records from 'records' to it.
        """
        cur    = conn.cursor()
        tbl    = SqliteStorage.recordTbl
        cur.execute('DROP TABLE IF EXISTS %s' % tbl)
        SqliteStorage.createTables(conn)
        fields = list(Record.fields.keys())
        flds   = ','.join(fields)
        hlds   = ','.join(['?'] * len(fields))
        sql    = 'INSERT INTO %s (%s) VALUES (%s)' % (tbl, flds, hlds)
        count  = 0
        for data in records:
            data = Record.convertFields(data.items(), False)
            vals = [data[k] for k in fields]
            cur.execute(sql, vals)
            count += 1
        conn.commit()
        conn.close()
        print('%s records inserted' % count)

def help(cate=None, ofile=sys.stdout):
    bname = os.path.basename(sys.argv[0])
    defaultMsg = "Usage: %s <command> [option [argument]]... [-F config]\n"
    defaultMsg += "       %s <command> --help\n"
    defaultMsg += "available commands: add, del, edit, list, push, fetch, clone, man\n"
    defaultMsg += """\nInitialization steps:

1. Create config file with content like the following,
   the default path of the config is ~/.logrc.
# Where to store the log data
dataDir = '/home/joshua/.log'
# Author name
authorName = 'Joshua Chen'
# Author email
authorEmail = 'iesugrace@gmail.com'

2. Run the command, if the config file is not placed at the
   default location, specify it with -F option. The directory
   structure will be created automatically.
# log add
# log list
# log add -F /path/to/config
"""
    defaultMsg = defaultMsg % ((bname,) * 2)

    addMsg = """
%s add
%s add -m message
%s add -m message < file
pipe | %s add -m message
%s add -g tag -t time -c scene -p people -m message
""" % ((bname,) * 5)

    listMsg = """
%s list                             -- list all
%s list ead1 0c46                   -- list some specific records
%s list -3                          -- the last three added/changed
%s list -t 2016                     -- in 2016
%s list -t 201601                   -- in Jan. 2016
%s list -t 20160107                 -- on Jan. 7th 2016
%s list -t -7                       -- the 7th day before
%s list -t 3:5                      -- 3th through 5th this month
%s list -t -2:today                 -- from the day before yesterday up to today
%s list -t -2:                      -- from the day before yesterday up to today
%s list -t 0703:0909                -- 3rd Jul. through 9th Sep. this year
%s list -t 20150101:today           -- from Jan. 1st 2015 up to today
%s list -t 20160101,3:5,-2,12       -- any day matches the comma separated list
%s list -mt :                       -- modify time is used instead of time
%s list -S '/home/'                 -- any field matches the regular expression
%s list -S '/home/i'                -- ignore case
%s list -S 'scene/home/i'           -- the scene field matches the regular expression
%s list -S<RE> -S<RE>               -- any of the REs matches any of the fields
%s list -S<RE> -S<RE> --all-match   -- each of the REs matches any of the fields
%s list -f '%%i: (%%t, %%mt) %%s'       -- specify the display format
%s list --sort time -r              -- sort by time, reverse
%s list -t 3:5 -S<RE>               -- match time and RE
""" % ((bname,) * 22)

    delMsg = """
Support to match logs using any listing options
%s del 297aacc                  -- delete log whose id starts with 297aacc
%s del -1                       -- delete the last log
%s del -t -1                    -- delete last day's logs
%s del -f 297aacc               -- delete without confirmation
%s del -i list-of-file          -- get log IDs from a file
""" % ((bname,) * 5)

    editMsg = """
Support to match logs using any listing options
%s edit 297aacc                  -- edit log whose id starts with 297aacc
%s edit -1                       -- edit the last log
%s edit -t -1                    -- edit last day's logs
""" % ((bname,) * 3)

    pushMsg = """
%s push origin                  -- push to remote 'origin'
%s push origin github           -- push to remote 'origin' and 'github'
%s push -a                      -- push to all remotes
""" % ((bname,) * 3)

    fetchMsg = """
%s fetch origin                 -- fetch from remote 'origin'
%s fetch origin github          -- fetch from remote 'origin' and 'github'
%s fetch -a                     -- fetch from all remotes
""" % ((bname,) * 3)

    cloneMsg = "%s clone <remote-url>" % bname

    manMsg = """
%s man unity                                    -- recreate sqlite using xml data
%s man export -f text -o dir [list-options]     -- export as text file to dir
%s man export -f xml -o dir [list-options]      -- export as xml file to dir
%s man export -f sqlite -o file [list-options]  -- export to a sqlite file
""" % ((bname,) * 4)

    if cate == 'add':
        msg = addMsg
    elif cate == 'list':
        msg = listMsg
    elif cate == 'del':
        msg = delMsg
    elif cate == 'edit':
        msg = editMsg
    elif cate == 'push':
        msg = pushMsg
    elif cate == 'fetch':
        msg = fetchMsg
    elif cate == 'clone':
        msg = cloneMsg
    elif cate == 'man':
        msg = manMsg
    else:
        msg = defaultMsg
    msg = msg.strip()
    print(msg, file=ofile)


def loadConfig():
    appConfigs = {}
    if '-F' in sys.argv:
        idx = sys.argv.index('-F')
        sys.argv.pop(idx)
        assert len(sys.argv) > idx, "need argument for -F option"
        path = sys.argv.pop(idx)
    else:
        path = None
    config = Config(path)
    for k, v in config.data.items():
        appConfigs[k] = v
    return appConfigs


if __name__ == '__main__':
    # load config from file
    try:
        appConfigs = loadConfig()
    except AssertionError:
        print(sys.exc_info()[1])
        exit(1)
    except:
        print(sys.exc_info()[1])
        help(ofile=sys.stderr)
        exit(1)

    ln = len(sys.argv)
    if ln == 1 or (ln == 2 and '--help' in sys.argv):
        help()
        exit(0)

    # start to work
    app = App(appConfigs)
    try:
        cmd = sys.argv[1]
        if cmd == 'add':
            app.add(sys.argv[2:])
        elif cmd == 'del':
            app.delete(sys.argv[2:])
        elif cmd == 'edit':
            app.edit(sys.argv[2:])
        elif cmd == 'list':
            app._list(sys.argv[2:])
        elif cmd == 'push':
            app.push(sys.argv[2:])
        elif cmd == 'fetch':
            app.fetch(sys.argv[2:])
        elif cmd == 'clone':
            app.clone(sys.argv[2:])
        elif cmd == 'man':
            app.man(sys.argv[2:])
        else:
            raise applib.InvalidCmdException('unrecognized command: %s' % cmd)
    except IndexError:
        print('argument error', file=sys.stderr)
        help(ofile=sys.stderr)
        exit(1)
    except BrokenPipeError:
        pass
    except (AssertionError,
            applib.InvalidTimeException,
            applib.InvalidReException,
            applib.InvalidFieldException,
            applib.InvalidCmdException,
            applib.NotTerminalException) as e:
        print(e, file=sys.stderr)
        exit(1)
